{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "id": "jKQFhU-jfraL",
    "colab_type": "code",
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 269.0
    },
    "outputId": "b8bf4d8b-3a7e-4d49-91ae-2a31942c5c0a"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-1-1036af063306>:47: BasicRNNCell.__init__ (from tensorflow.python.ops.rnn_cell_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "This class is equivalent as tf.keras.layers.SimpleRNNCell, and will be replaced by that in Tensorflow 2.0.\n",
      "Epoch: 1000 cost = 0.001075\n",
      "Epoch: 2000 cost = 0.000416\n",
      "Epoch: 3000 cost = 0.000238\n",
      "Epoch: 4000 cost = 0.000062\n",
      "Epoch: 5000 cost = 0.000021\n",
      "test\n",
      "man -> women\n",
      "mans -> women\n",
      "king -> queen\n",
      "black -> white\n",
      "upp -> down\n"
     ]
    }
   ],
   "source": [
    "'''\n",
    "  code by Tae Hwan Jung(Jeff Jung) @graykode\n",
    "  reference : https://github.com/golbin/TensorFlow-Tutorials/blob/master/10%20-%20RNN/03%20-%20Seq2Seq.py\n",
    "'''\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "\n",
    "tf.reset_default_graph()\n",
    "# S: Symbol that shows starting of decoding input\n",
    "# E: Symbol that shows starting of decoding output\n",
    "# P: Symbol that will fill in blank sequence if current batch data size is short than time steps\n",
    "\n",
    "char_arr = [c for c in 'SEPabcdefghijklmnopqrstuvwxyz']\n",
    "num_dic = {n: i for i, n in enumerate(char_arr)}\n",
    "\n",
    "seq_data = [['man', 'women'], ['black', 'white'], ['king', 'queen'], ['girl', 'boy'], ['up', 'down'], ['high', 'low']]\n",
    "\n",
    "# Seq2Seq Parameter\n",
    "n_step = 5\n",
    "n_hidden = 128\n",
    "n_class = len(num_dic) # number of class(=number of vocab)\n",
    "\n",
    "def make_batch(seq_data):\n",
    "    input_batch, output_batch, target_batch = [], [], []\n",
    "\n",
    "    for seq in seq_data:\n",
    "        for i in range(2):\n",
    "            seq[i] = seq[i] + 'P' * (n_step - len(seq[i]))\n",
    "\n",
    "        input = [num_dic[n] for n in seq[0]]\n",
    "        output = [num_dic[n] for n in ('S' + seq[1])]\n",
    "        target = [num_dic[n] for n in (seq[1] + 'E')]\n",
    "\n",
    "        input_batch.append(np.eye(n_class)[input])\n",
    "        output_batch.append(np.eye(n_class)[output])\n",
    "\n",
    "        target_batch.append(target)\n",
    "\n",
    "    return input_batch, output_batch, target_batch\n",
    "\n",
    "# Model\n",
    "enc_input = tf.placeholder(tf.float32, [None, None, n_class]) # [batch_size, max_len(=encoder_step), n_class]\n",
    "dec_input = tf.placeholder(tf.float32, [None, None, n_class]) # [batch_size, max_len+1(=decoder_step) (becase of 'S' or 'E'), n_class]\n",
    "targets = tf.placeholder(tf.int64, [None, None]) # [batch_size, max_len+1], not one-hot\n",
    "\n",
    "with tf.variable_scope('encode'):\n",
    "    enc_cell = tf.nn.rnn_cell.BasicRNNCell(n_hidden)\n",
    "    enc_cell = tf.nn.rnn_cell.DropoutWrapper(enc_cell, output_keep_prob=0.5)\n",
    "    _, enc_states = tf.nn.dynamic_rnn(enc_cell, enc_input, dtype=tf.float32)\n",
    "    # encoder state will go to decoder initial_state, enc_states : [batch_size, n_hidden(=128)]\n",
    "\n",
    "with tf.variable_scope('decode'):\n",
    "    dec_cell = tf.nn.rnn_cell.BasicRNNCell(n_hidden)\n",
    "    dec_cell = tf.nn.rnn_cell.DropoutWrapper(dec_cell, output_keep_prob=0.5)\n",
    "    outputs, _ = tf.nn.dynamic_rnn(dec_cell, dec_input, initial_state=enc_states, dtype=tf.float32)\n",
    "    # outputs : [batch_size, max_len+1, n_hidden(=128)]\n",
    "\n",
    "model = tf.layers.dense(outputs, n_class, activation=None) # model : [batch_size, max_len+1, n_class]\n",
    "\n",
    "cost = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(logits=model, labels=targets))\n",
    "optimizer = tf.train.AdamOptimizer(0.001).minimize(cost)\n",
    "\n",
    "# Training\n",
    "sess = tf.Session()\n",
    "sess.run(tf.global_variables_initializer())\n",
    "input_batch, output_batch, target_batch = make_batch(seq_data)\n",
    "\n",
    "for epoch in range(5000):\n",
    "    _, loss = sess.run([optimizer, cost], feed_dict={enc_input: input_batch, dec_input: output_batch, targets: target_batch})\n",
    "    if (epoch + 1)%1000 == 0:\n",
    "        print('Epoch:', '%04d' % (epoch + 1), 'cost =', '{:.6f}'.format(loss))\n",
    "\n",
    "# Test\n",
    "def translate(word):\n",
    "    seq_data = [word, 'P' * len(word)]\n",
    "\n",
    "    input_batch, output_batch, _ = make_batch([seq_data])\n",
    "    prediction = tf.argmax(model, 2)\n",
    "\n",
    "    result = sess.run(prediction, feed_dict={enc_input: input_batch, dec_input: output_batch})\n",
    "\n",
    "    decoded = [char_arr[i] for i in result[0]]\n",
    "    end = decoded.index('E')\n",
    "    translated = ''.join(decoded[:end])\n",
    "\n",
    "    return translated.replace('P','')\n",
    "\n",
    "print('test')\n",
    "print('man ->', translate('man'))\n",
    "print('mans ->', translate('mans'))\n",
    "print('king ->', translate('king'))\n",
    "print('black ->', translate('black'))\n",
    "print('upp ->', translate('upp'))"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "name": "Seq2Seq-Tensor.ipynb",
   "version": "0.3.2",
   "provenance": [],
   "collapsed_sections": []
  },
  "kernelspec": {
   "name": "python3",
   "display_name": "Python 3"
  },
  "accelerator": "GPU"
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
